#!/bin/bash
#这个代码完成所有的利用lsa计算相似度的操作
#分词
if [ $# -lt 1 ];
then
    echo "usage: lsa_related.sh agen_id";
    exit
else
    age_id=$1
fi
if [ $# -eq 2 ];
then
    startId=$2    
else
    startId=1
fi


svdServer=172.16.10.10
svdUser=lvhl
preProcessPath="../preProcess"
postProcessPath="../postProcess"
getSimPath="../getsim"
splitWordPath="../splitWord"
linkAnalysePath="../link_analyse"
dataDir="/data/lvhongliang/data/ask"
questeionNormalFile="/data/lvhongliang/data/ask/all_question_normal"
questeionDelFile="/data/lvhongliang/data/ask/all_question_deleted"

#
#age_id: 准备怀孕 1\n";
#        怀孕期   2\n";
#        0-1岁    3\n";
#        1-3岁    4\n";
#        3-6岁    5\n";
#        情感家庭 477\n";
#        生活消费 488\n";
#age_id=5

#获得document的数量
curDataDir=$dataDir"/"$age_id
docMapFile=$curDataDir"/doc_id.map"
docNum=`wc -l $docMapFile | awk '{print $1}'`
tfidfFile=$curDataDir"/doc_term_tf_idf.txt"
latentVectorFile=$curDataDir"/doc_latent_vectors.txt"
clusterMap=$curDataDir"/assignCid.txt" 
clusterSet=$curDataDir"/clusters.txt"

#13.计算相似度存储在数据库中和文件中
#(1)准备config file
echo "step 13: calculate the similarity and store to file and database"
echo "step 13.1 create and truncate table"
#mysql -h 172.16.10.10 -u babytree -pbabytree ask -e "create table sim_tfidf_$age_id like sim_tfidf;"
#mysql -h 172.16.10.10 -u babytree -pbabytree ask -e "truncate table sim_tfidf_$age_id;"

configFile=$getSimPath"/config/"$age_id"_config_tfidf_savesimdb.txt"
docSimFile=$curDataDir"/doc_sim_tfidf.txt"
#准备将id映射到docId的map文件
id_docMapFile=$curDataDir"/id_doc.map"
awk '{print $2,$1}' $docMapFile > $id_docMapFile

source $getSimPath/genGetSimConfig.sh
if [ ! -f $docSimFile ];then
     writeSimConfig $configFile $docNum $tfidfFile $docSimFile  $startId $docNum $clusterMap $clusterSet "sim_tfidf_"$age_id $id_docMapFile
    echo "***********begin calculating the tfidf similarity !*******"
    #(2)计算
    $getSimPath/getSim_tfidf -c $configFile #1>$curDataDir"/1_getSim.txt" 2>$curDataDir"/2_getSim.txt"
    if [ $? -ne 0 ] ; then
        echo "error"
        exit 1
    fi
fi
#14. 统计入链
echo "step 14: state the inlink of every document"
#trim 掉 doc_sim.txt中的weight信息
$linkAnalysePath/deleteWeight.sh $docSimFile > $curDataDir"/doc_sim_tfidf.trim"
$linkAnalysePath/stat_in_link.sh $curDataDir"/doc_sim_tfidf.trim" > $curDataDir"/tfidf_inlink.txt"
if [ $? -ne 0 ] ; then
    echo "error"
    exit 1
fi

#15. 通过inlink降权得到最终结果
echo "step 15: lower by inlink and get the final result"
inlinkConfigFile=$getSimPath"/config/"$age_id"_config_updateNeigh_tfidf.txt"
writeInlinkLowConfig $inlinkConfigFile $docNum $curDataDir"/tfidf_inlink.txt" $curDataDir"/doc_sim_tfidf.trim" 10 "sim_tfidf_$age_id";
$getSimPath/updateNeighbor -c $inlinkConfigFile 1>$curDataDir"/1_update_inlink.txt" 2> $curDataDir"/2_update_inlink.txt" 
if [ $? -ne 0 ] ; then
    echo "error"
    exit 1
fi

#16. 将转换后的id还原 
echo "step 16: restore the mappedId"
inlinkOriFile=$curDataDir"/sim_tfidf_{$age_id}_step_9.ori" 
$postProcessPath/restoreToOriginalDocId.sh  $docSimFile $docSimFile".ori"  $docMapFile
$postProcessPath/restoreToOriginalDocId.sh  $curDataDir"/sim_tfidf_{$age_id}_step_9" $inlinkOriFile $docMapFile
if [ $? -ne 0 ] ; then
    echo "error"
    exit 1
fi
#17. deleteWeight
$linkAnalysePath/deleteWeight.sh $docSimFile".ori" > $docSimFile".ori.trim"
$linkAnalysePath/deleteWeight.sh $inlinkOriFile > $inlinkOriFile".trim"

#18. bzip2 file
bzip2 $docSimFile".ori.trim"
bzip2 $inlinkOriFile".trim" 

#19. scp file
echo "step 19: scp the *.ori.trim to s1"
scp $inlinkOriFile".trim.bz2"  lvhongliang@s1:related_data_full/

#20. unzip file
echo "step 20: unzip the *.ori.trim.bz2 to *.ori.trim on s1"
ssh lvhongliang@s1 "bunzip2 ~/related_data_full/${inlinkOriFile}.trim.bz2"

echo "Have finished all the steps, enjoy it!"
exit 0
